{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "override USE_NET <class 'src.networks.ExpTripletNetwork'>\n",
      "override DATASET_PROC_METHOD_TRAIN Rescale\n",
      "override DATASET_PROC_METHOD_VAL Rescale\n",
      "override MAX_CATEGORY_NUM 64\n",
      "override IMAGE_EMBED_SIZE 512\n",
      "override NEGATIVE_SAMPLE_WITH_TYPE True\n",
      "override LEARNED_FIELD_EMBED False\n",
      "override LEARNED_FIELD_BIAS False\n",
      "override TRIPLET_MARGIN 0.2\n",
      "override LEARNED_METRICS True\n",
      "override WEIGHT_TRIPLET 1.0\n",
      "override WEIGHT_L1_MASK 0.1\n",
      "override WEIGHT_L2_GENERAL_EMB 0.1\n",
      "override USE_PRETRAINED_WORD_EMBEDDING False\n",
      "override WORD_EMBED_SIZE 300\n",
      "override MAX_VOCAB_SIZE 300\n",
      "override OUTFIT_NAME_PAD_NUM 10\n",
      "override NUM_EPOCH 70\n",
      "override LEARNING_RATE 0.0001\n",
      "override LEARNING_RATE_DECAY 0.95\n",
      "override BATCH_SIZE 64\n",
      "override SAVE_EVERY_STEPS 10000\n",
      "override SAVE_EVERY_EPOCHS 1\n",
      "override VAL_WHILE_TRAIN True\n",
      "override VAL_FASHION_COMP_FILE fashion_compatibility_small.txt\n",
      "override VAL_FITB_FILE fill_in_blank_test_small.json\n",
      "override VAL_BATCH_SIZE 8\n",
      "override VAL_EVERY_STEPS 1000\n",
      "override VAL_EVERY_EPOCHS 1\n",
      "override VAL_START_EPOCH 1\n",
      "override device cuda:0\n",
      "override TRAIN_DIR runs/src.conf.fixm_gm/11-06 01:17:53\n",
      "override VAL_DIR runs/src.conf.fixm_gm/11-06 01:17:53\n",
      "override MODEL_NAME src.conf.fixm_gm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "from src.const import base_path\n",
    "import numpy as np\n",
    "import cv2\n",
    "from torchvision import transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import skimage\n",
    "from src import const\n",
    "import json\n",
    "import os\n",
    "import nltk\n",
    "from src.utils import load_json, build_vocab, Vocab\n",
    "from src.base_networks import *\n",
    "from src.networks import *\n",
    "from src.dataset import *\n",
    "from torch import nn\n",
    "import torchvision\n",
    "from torch.nn import functional as F\n",
    "from src.utils import merge_const\n",
    "import random\n",
    "merge_const('src.conf.fixm_gm')\n",
    "const.BATCH_SIZE = 2\n",
    "const.device = 'cpu'\n",
    "class _(object):\n",
    "    pass\n",
    "self = _()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building positive pairs...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f6fec1bcdd8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 349, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 328, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 337, in get\n",
      "    return _ForkingPickler.loads(res)\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/multiprocessing/reduction.py\", line 182, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/home/hzy/anaconda3/lib/python3.6/multiprocessing/reduction.py\", line 153, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    }
   ],
   "source": [
    "train_set = load_json(os.path.join(const.base_path, 'train_no_dup.json'))\n",
    "valid_set = load_json(os.path.join(const.base_path, 'valid_no_dup.json'))\n",
    "test_set = load_json(os.path.join(const.base_path, 'test_no_dup.json'))\n",
    "vocab = build_vocab(train_set)\n",
    "train_dataset = PolyvoreTripletDataset(train_set, const.DATASET_PROC_METHOD_TRAIN, vocab)\n",
    "train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=const.BATCH_SIZE, shuffle=True, num_workers=4)\n",
    "sample = iter(train_dataloader).next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All categories: {'Upper': 0, 'All': 1, 'Lower': 2, 'Others': 3, 'Bags': 4, 'Shoes': 5, 'Accessories': 6, 'Hats': 7, 'Glasses': 8, 'Necklaces': 9, 'Earrings': 10, 'Rings': 11, 'Makeup': 12, 'Furniture': 13, 'Underclothes': 14, 'Tech': 15}\n",
      "Number of categories 16\n"
     ]
    }
   ],
   "source": [
    "const.CATEGORY_CLASS_FILE = 'category_class_coarse.txt'\n",
    "with open(os.path.join(const.base_path, const.CATEGORY_CLASS_FILE)) as f:\n",
    "    name2real_id = {}\n",
    "    cate2real_id = {}\n",
    "    cnt = 0\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if line == \"\":\n",
    "            continue\n",
    "        line = line.split(' ')\n",
    "        real_name = line[0].strip()\n",
    "        cate_id = int(line[1].strip())\n",
    "        if real_name not in name2real_id:\n",
    "            real_id = cnt\n",
    "            name2real_id[real_name] = real_id\n",
    "            cnt += 1\n",
    "        else:\n",
    "            real_id = name2real_id[real_name]\n",
    "        cate2real_id[cate_id] = real_id\n",
    "    print (\"All categories:\", name2real_id)\n",
    "    print (\"Number of categories\", len(name2real_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2: 0,\n",
       " 3: 1,\n",
       " 4: 1,\n",
       " 5: 1,\n",
       " 6: 1,\n",
       " 7: 2,\n",
       " 8: 2,\n",
       " 9: 2,\n",
       " 10: 2,\n",
       " 11: 0,\n",
       " 15: 0,\n",
       " 17: 0,\n",
       " 18: 0,\n",
       " 19: 0,\n",
       " 21: 0,\n",
       " 23: 0,\n",
       " 24: 0,\n",
       " 25: 0,\n",
       " 26: 0,\n",
       " 27: 2,\n",
       " 28: 2,\n",
       " 29: 2,\n",
       " 30: 2,\n",
       " 31: 3,\n",
       " 33: 3,\n",
       " 35: 4,\n",
       " 36: 4,\n",
       " 37: 4,\n",
       " 38: 4,\n",
       " 39: 4,\n",
       " 40: 4,\n",
       " 41: 5,\n",
       " 42: 5,\n",
       " 43: 5,\n",
       " 46: 5,\n",
       " 47: 5,\n",
       " 48: 5,\n",
       " 49: 5,\n",
       " 50: 5,\n",
       " 51: 6,\n",
       " 52: 3,\n",
       " 53: 6,\n",
       " 55: 7,\n",
       " 56: 6,\n",
       " 57: 8,\n",
       " 58: 6,\n",
       " 59: 3,\n",
       " 60: 6,\n",
       " 61: 6,\n",
       " 62: 9,\n",
       " 64: 10,\n",
       " 65: 11,\n",
       " 343: 0,\n",
       " 67: 6,\n",
       " 68: 3,\n",
       " 69: 6,\n",
       " 71: 3,\n",
       " 74: 3,\n",
       " 75: 3,\n",
       " 76: 3,\n",
       " 77: 3,\n",
       " 78: 3,\n",
       " 85: 3,\n",
       " 93: 12,\n",
       " 94: 12,\n",
       " 95: 12,\n",
       " 96: 12,\n",
       " 97: 12,\n",
       " 98: 12,\n",
       " 99: 12,\n",
       " 101: 3,\n",
       " 102: 3,\n",
       " 104: 0,\n",
       " 105: 6,\n",
       " 106: 6,\n",
       " 107: 6,\n",
       " 108: 3,\n",
       " 110: 3,\n",
       " 113: 13,\n",
       " 115: 13,\n",
       " 116: 13,\n",
       " 118: 13,\n",
       " 120: 13,\n",
       " 122: 13,\n",
       " 123: 13,\n",
       " 124: 13,\n",
       " 126: 13,\n",
       " 127: 13,\n",
       " 129: 13,\n",
       " 130: 13,\n",
       " 132: 13,\n",
       " 135: 13,\n",
       " 136: 13,\n",
       " 139: 13,\n",
       " 140: 13,\n",
       " 141: 13,\n",
       " 143: 13,\n",
       " 144: 13,\n",
       " 4241: 13,\n",
       " 4242: 13,\n",
       " 147: 13,\n",
       " 4244: 13,\n",
       " 150: 13,\n",
       " 4247: 13,\n",
       " 4248: 13,\n",
       " 153: 13,\n",
       " 154: 13,\n",
       " 155: 13,\n",
       " 157: 13,\n",
       " 4254: 13,\n",
       " 159: 13,\n",
       " 160: 13,\n",
       " 4257: 13,\n",
       " 162: 13,\n",
       " 163: 13,\n",
       " 164: 13,\n",
       " 166: 13,\n",
       " 167: 13,\n",
       " 168: 13,\n",
       " 169: 13,\n",
       " 170: 13,\n",
       " 4267: 13,\n",
       " 172: 13,\n",
       " 173: 13,\n",
       " 174: 13,\n",
       " 175: 13,\n",
       " 176: 13,\n",
       " 177: 13,\n",
       " 178: 13,\n",
       " 179: 13,\n",
       " 180: 13,\n",
       " 181: 13,\n",
       " 182: 13,\n",
       " 184: 12,\n",
       " 185: 12,\n",
       " 186: 12,\n",
       " 187: 12,\n",
       " 188: 12,\n",
       " 189: 12,\n",
       " 190: 12,\n",
       " 191: 12,\n",
       " 192: 12,\n",
       " 193: 12,\n",
       " 194: 12,\n",
       " 195: 12,\n",
       " 4292: 12,\n",
       " 197: 12,\n",
       " 198: 12,\n",
       " 199: 12,\n",
       " 200: 12,\n",
       " 201: 12,\n",
       " 202: 12,\n",
       " 203: 12,\n",
       " 204: 12,\n",
       " 205: 12,\n",
       " 206: 12,\n",
       " 207: 12,\n",
       " 208: 12,\n",
       " 209: 12,\n",
       " 211: 12,\n",
       " 213: 12,\n",
       " 214: 12,\n",
       " 216: 12,\n",
       " 217: 12,\n",
       " 218: 12,\n",
       " 219: 12,\n",
       " 220: 12,\n",
       " 221: 12,\n",
       " 222: 12,\n",
       " 223: 12,\n",
       " 224: 12,\n",
       " 225: 12,\n",
       " 226: 12,\n",
       " 227: 12,\n",
       " 228: 12,\n",
       " 229: 12,\n",
       " 231: 12,\n",
       " 236: 0,\n",
       " 237: 2,\n",
       " 238: 2,\n",
       " 239: 2,\n",
       " 240: 2,\n",
       " 241: 2,\n",
       " 243: 1,\n",
       " 244: 1,\n",
       " 245: 14,\n",
       " 246: 14,\n",
       " 247: 14,\n",
       " 248: 14,\n",
       " 249: 14,\n",
       " 250: 14,\n",
       " 251: 14,\n",
       " 252: 0,\n",
       " 253: 0,\n",
       " 254: 0,\n",
       " 255: 0,\n",
       " 256: 0,\n",
       " 257: 14,\n",
       " 258: 4,\n",
       " 259: 4,\n",
       " 260: 4,\n",
       " 261: 5,\n",
       " 262: 5,\n",
       " 263: 5,\n",
       " 264: 5,\n",
       " 265: 5,\n",
       " 266: 5,\n",
       " 267: 5,\n",
       " 268: 5,\n",
       " 270: 6,\n",
       " 271: 0,\n",
       " 272: 0,\n",
       " 273: 0,\n",
       " 275: 0,\n",
       " 276: 0,\n",
       " 277: 0,\n",
       " 278: 2,\n",
       " 279: 2,\n",
       " 280: 2,\n",
       " 281: 0,\n",
       " 282: 2,\n",
       " 283: 2,\n",
       " 284: 2,\n",
       " 285: 2,\n",
       " 286: 0,\n",
       " 287: 2,\n",
       " 288: 2,\n",
       " 289: 0,\n",
       " 290: 4,\n",
       " 291: 5,\n",
       " 292: 5,\n",
       " 293: 5,\n",
       " 294: 5,\n",
       " 295: 5,\n",
       " 296: 5,\n",
       " 297: 5,\n",
       " 298: 5,\n",
       " 299: 6,\n",
       " 300: 6,\n",
       " 301: 6,\n",
       " 302: 6,\n",
       " 303: 7,\n",
       " 304: 8,\n",
       " 305: 6,\n",
       " 306: 6,\n",
       " 307: 3,\n",
       " 308: 3,\n",
       " 309: 0,\n",
       " 310: 2,\n",
       " 311: 3,\n",
       " 313: 13,\n",
       " 314: 13,\n",
       " 315: 14,\n",
       " 316: 13,\n",
       " 317: 13,\n",
       " 318: 4,\n",
       " 319: 12,\n",
       " 320: 3,\n",
       " 321: 3,\n",
       " 4426: 8,\n",
       " 4428: 8,\n",
       " 333: 3,\n",
       " 334: 12,\n",
       " 335: 12,\n",
       " 4432: 3,\n",
       " 4433: 13,\n",
       " 338: 12,\n",
       " 339: 12,\n",
       " 340: 12,\n",
       " 4437: 12,\n",
       " 342: 12,\n",
       " 4439: 13,\n",
       " 4440: 13,\n",
       " 4441: 3,\n",
       " 4442: 3,\n",
       " 4443: 3,\n",
       " 4445: 3,\n",
       " 4446: 13,\n",
       " 4447: 8,\n",
       " 4448: 3,\n",
       " 4449: 13,\n",
       " 4450: 13,\n",
       " 4451: 13,\n",
       " 4452: 2,\n",
       " 4454: 0,\n",
       " 4455: 0,\n",
       " 4456: 0,\n",
       " 4457: 0,\n",
       " 4458: 2,\n",
       " 4459: 2,\n",
       " 4460: 6,\n",
       " 4461: 4,\n",
       " 4462: 4,\n",
       " 4463: 4,\n",
       " 4464: 5,\n",
       " 4465: 5,\n",
       " 4466: 6,\n",
       " 4467: 6,\n",
       " 4468: 6,\n",
       " 4470: 6,\n",
       " 4472: 6,\n",
       " 4473: 6,\n",
       " 4474: 4,\n",
       " 4475: 12,\n",
       " 4476: 6,\n",
       " 4477: 6,\n",
       " 4478: 3,\n",
       " 4479: 3,\n",
       " 4480: 3,\n",
       " 4481: 3,\n",
       " 4482: 3,\n",
       " 4483: 11,\n",
       " 4484: 13,\n",
       " 4485: 13,\n",
       " 4486: 3,\n",
       " 4487: 13,\n",
       " 4488: 13,\n",
       " 4489: 13,\n",
       " 4490: 13,\n",
       " 4492: 13,\n",
       " 4493: 13,\n",
       " 4495: 0,\n",
       " 4496: 0,\n",
       " 4497: 0,\n",
       " 4498: 0,\n",
       " 4499: 13,\n",
       " 4500: 13,\n",
       " 4501: 13,\n",
       " 4502: 13,\n",
       " 4503: 13,\n",
       " 4504: 13,\n",
       " 4505: 13,\n",
       " 4506: 13,\n",
       " 4507: 13,\n",
       " 4508: 13,\n",
       " 4509: 13,\n",
       " 4510: 13,\n",
       " 4511: 13,\n",
       " 4512: 13,\n",
       " 4513: 13,\n",
       " 4514: 13,\n",
       " 4516: 13,\n",
       " 4517: 0,\n",
       " 4518: 2,\n",
       " 4520: 4,\n",
       " 4521: 4,\n",
       " 4522: 5,\n",
       " 4523: 6,\n",
       " 4524: 9,\n",
       " 4525: 11,\n",
       " 4526: 6,\n",
       " 4438: 13,\n",
       " 4240: 13,\n",
       " 146: 13,\n",
       " 148: 13,\n",
       " 4246: 13,\n",
       " 151: 13,\n",
       " 152: 13,\n",
       " 949: 13,\n",
       " 161: 13,\n",
       " 4258: 13,\n",
       " 171: 13,\n",
       " 4269: 13,\n",
       " 4276: 13,\n",
       " 196: 12,\n",
       " 3336: 3,\n",
       " 1605: 1,\n",
       " 1606: 1,\n",
       " 1607: 1,\n",
       " 1967: 15,\n",
       " 332: 2,\n",
       " 4429: 13,\n",
       " 4430: 13,\n",
       " 4431: 13,\n",
       " 336: 12,\n",
       " 337: 12,\n",
       " 4434: 3,\n",
       " 4435: 3,\n",
       " 4436: 3,\n",
       " 341: 0}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate2real_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.to_real_cateid(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
